

import scrapy
import json
import time
import hashlib, re
from urllib.parse import urlencode
from taobao.items import TaobaoItem

# 运行分布式爬虫 命令
# scrapy runspider phone.py

# ------1 导入分布式爬虫类
from scrapy_redis.spiders import RedisSpider

#  ------2 继承分布式爬虫类
class PhoneSpider(RedisSpider):
    name = "phone"

    #  ------3 注销 start_urls 和 allowed_domains
    # allowed_domains = ["h5api.m.taobao.com"]
    # start_urls = ["https://h5api.m.taobao.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/"]

    #  ------4 设置 redis_key
    redis_key = 'py21'

    #  ------5 设置 __init__
    def __init__(self, *args, **kwargs):
        domain = kwargs.pop('domain', '')
        self.allowed_domains = list(filter(None, domain.split(',')))
        super(PhoneSpider, self).__init__(*args, **kwargs)



    def md5_encrypt(self, input_string):
        # 创建一个md5 hasher对象
        hasher = hashlib.md5()

        # 编码输入字符串，因为hashlib需要字节类型的数据
        input_bytes = input_string.encode('utf-8')

        # 对输入数据进行MD5加密
        hasher.update(input_bytes)

        # 获取十六进制格式的加密结果
        md5_result = hasher.hexdigest()

        return md5_result


    def get_params(self):

        data = "{\"appId\":\"34385\",\"params\":\"{\\\"device\\\":\\\"HMA-AL00\\\",\\\"isBeta\\\":\\\"false\\\",\\\"grayHair\\\":\\\"false\\\",\\\"from\\\":\\\"nt_history\\\",\\\"brand\\\":\\\"HUAWEI\\\",\\\"info\\\":\\\"wifi\\\",\\\"index\\\":\\\"4\\\",\\\"rainbow\\\":\\\"\\\",\\\"schemaType\\\":\\\"auction\\\",\\\"elderHome\\\":\\\"false\\\",\\\"isEnterSrpSearch\\\":\\\"true\\\",\\\"newSearch\\\":\\\"false\\\",\\\"network\\\":\\\"wifi\\\",\\\"subtype\\\":\\\"\\\",\\\"hasPreposeFilter\\\":\\\"false\\\",\\\"prepositionVersion\\\":\\\"v2\\\",\\\"client_os\\\":\\\"Android\\\",\\\"gpsEnabled\\\":\\\"false\\\",\\\"searchDoorFrom\\\":\\\"srp\\\",\\\"debug_rerankNewOpenCard\\\":\\\"false\\\",\\\"homePageVersion\\\":\\\"v7\\\",\\\"searchElderHomeOpen\\\":\\\"false\\\",\\\"search_action\\\":\\\"initiative\\\",\\\"sugg\\\":\\\"_4_1\\\",\\\"sversion\\\":\\\"13.6\\\",\\\"style\\\":\\\"list\\\",\\\"ttid\\\":\\\"600000@taobao_pc_10.7.0\\\",\\\"needTabs\\\":\\\"true\\\",\\\"areaCode\\\":\\\"CN\\\",\\\"vm\\\":\\\"nw\\\",\\\"countryNum\\\":\\\"156\\\",\\\"m\\\":\\\"pc\\\",\\\"page\\\":2,\\\"n\\\":48,\\\"q\\\":\\\"%E6%89%8B%E6%9C%BA\\\",\\\"qSource\\\":\\\"url\\\",\\\"pageSource\\\":\\\"a21bo.jianhua/a.201867-main.d3_second.f4d92a89t2Cyxb\\\",\\\"channelSrp\\\":\\\"\\\",\\\"tab\\\":\\\"all\\\",\\\"pageSize\\\":\\\"48\\\",\\\"totalPage\\\":\\\"100\\\",\\\"totalResults\\\":\\\"75812\\\",\\\"sourceS\\\":\\\"0\\\",\\\"sort\\\":\\\"_coefp\\\",\\\"bcoffset\\\":\\\"-9\\\",\\\"ntoffset\\\":\\\"13\\\",\\\"filterTag\\\":\\\"\\\",\\\"service\\\":\\\"\\\",\\\"prop\\\":\\\"\\\",\\\"loc\\\":\\\"\\\",\\\"start_price\\\":null,\\\"end_price\\\":null,\\\"startPrice\\\":null,\\\"endPrice\\\":null,\\\"categoryp\\\":\\\"\\\",\\\"ha3Kvpairs\\\":null,\\\"myCNA\\\":\\\"8tabH2SLTDYCAW8EOAqgp1/9\\\",\\\"screenResolution\\\":\\\"1536x864\\\",\\\"userAgent\\\":\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36\\\",\\\"couponUnikey\\\":\\\"\\\",\\\"subTabId\\\":\\\"\\\",\\\"np\\\":\\\"\\\",\\\"clientType\\\":\\\"h5\\\",\\\"isNewDomainAb\\\":\\\"false\\\",\\\"forceOldDomain\\\":\\\"false\\\"}\"}"
        t = int(time.time()*1000)
        sign = self.md5_encrypt("a3978e9e8bf8b128476e00d0e3bdd3c2" + "&" + str(t) + "&" + "12574478" + "&" + data)
        params = {
            "jsv": "2.7.4",
            "appKey": "12574478",
            "t": t,
            "sign": sign,
            "api": "mtop.relationrecommend.wirelessrecommend.recommend",
            "v": "2.0",
            "timeout": "10000",
            "type": "jsonp",
            "dataType": "jsonp",
            "callback": "mtopjsonp22",
            "data": data,
        }
        return params


    def start_requests(self):
        cookies = {
            't': '37fd7edb0e01d2c036dd465707752a56',
            'cna': '8tabH2SLTDYCAW8EOAqgp1/9',
            'xlly_s': '1',
            'thw': 'cn',
            'cookie2': '16aeb214ee57c7d8676015f0e0e39ef4',
            '_tb_token_': '7f535ee360e6f',
            'mtop_partitioned_detect': '1',
            '_m_h5_tk': 'a3978e9e8bf8b128476e00d0e3bdd3c2_1754982955554',
            '_m_h5_tk_enc': '683090e6354bab025cba0896dec22cf3',
            'bxuab': '0',
            'sca': '462f85cb',
            '_samesite_flag_': 'true',
            '3PcFlag': '1754974328113',
            'sgcookie': 'E100HKx7EXGOBn3rtpfXDxaCEyHGEomMS2uSqL%2FdpGfrzAEgMzESLHo5qOl%2FghoFcpb4SaSKg7dE3oIlC8kHJ9mTUg7h0MzQ6Z5e%2BPbXX3iJCwz5HshX6nIiKx92Q5HRKo4B',
            'wk_cookie2': '10d5c1cb6f00327024c00068b52fe954',
            'wk_unb': 'UUphzpYpo1cAMK%2F4JQ%3D%3D',
            'unb': '2204117959774',
            'uc1': 'cookie16=U%2BGCWk%2F74Mx5tgzv3dWpnhjPaQ%3D%3D&pas=0&cookie21=URm48syIYn73&cookie14=UoYbz9K30cCPfQ%3D%3D&existShop=false&cookie15=W5iHLLyFOGW7aA%3D%3D',
            'uc3': 'nk2=F5RARU2%2BMlM9w4Y%3D&lg2=WqG3DMC9VAQiUQ%3D%3D&vt3=F8dD2fjA1fnugzZMUnc%3D&id2=UUphzpYpo1cAMK%2F4JQ%3D%3D',
            'csg': 'dff9be49',
            'lgc': 'tb577868117',
            'cancelledSubSites': 'empty',
            'cookie17': 'UUphzpYpo1cAMK%2F4JQ%3D%3D',
            'dnk': 'tb577868117',
            'skt': 'a61bde99f4b54bff',
            'existShop': 'MTc1NDk3NDM0OQ%3D%3D',
            'uc4': 'nk4=0%40FY4L6b71QaBNOpVcobytGOi0rZ2M0A%3D%3D&id4=0%40U2grFbWywot0k%2FVccXZwMZyCVqyl6km0',
            'tracknick': 'tb577868117',
            '_cc_': 'Vq8l%2BKCLiw%3D%3D',
            '_l_g_': 'Ug%3D%3D',
            'sg': '749',
            '_nk_': 'tb577868117',
            'cookie1': 'VWZ4GOfwjCv3ijD87ymhA53ch4D%2BHij9VcYzw1mgmr0%3D',
            'tfstk': 'gFXrdFXOd8ePJe0bFZpE7Ry2LDpJupz_U9TBxMjHVUYkPMxHLZbQ2Us3ywWeoMC72Ba-Y0BV7vM7y7InLp935Pw_CgIWpp460kPj3UKdmeqXZ4Ske0v35PwsGmA8PpbIbOK2mjx9o3xH-eq2n3tpqpAHqiq2xHgotYbhmo8vj3DHKHjDmh-KKpvh-iq2vExkKQbhmo-poecfeeYs3hRumVqIt_cEEQYGqEknQcKkZC29uvDh33S24gctKvX24QXLpaVgTQ1hXCB5VJkvFG5Dn373IzvcjsS6aTzZFedF3a5FPkGp06ShBLpZx8j2UURGUQnKQNjVx1OVhl29ULYlOLIIbzsVUasR3MiniLJ5gCWyIR3kRssNLHWLJ-7Nq1bhTglmJnAcT9Cr-XRkDn8_0or5sbGcCTHO7XhpwsK25uE-9XdkDn8_0olK9QCv0FZ82',
            'isg': 'BK2tXkwiuZYTdlL0t5HGtfOpvEknCuHcMr2HU--ypsSmZswYtlshrs5wVDqAZvmU',
        }

        headers = {
            "accept": "*/*",
            "accept-language": "zh-CN,zh;q=0.9",
            "cache-control": "no-cache",
            "pragma": "no-cache",
            "referer": "https://s.taobao.com/search?page=1&q=%E6%89%8B%E6%9C%BA&spm=a21bo.jianhua%2Fa.201867-main.d3_second.f4d92a89t2Cyxb&tab=all",
            "sec-ch-ua": "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Windows\"",
            "sec-fetch-dest": "script",
            "sec-fetch-mode": "no-cors",
            "sec-fetch-site": "same-site",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
        }

        # 获取 请求加密参数
        params = self.get_params()
        base_url = "https://h5api.m.taobao.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/"
        # 拼接 URL（GET 请求）
        url_with_params = f"{base_url}?{urlencode(params)}"

        yield scrapy.Request(
            url=url_with_params,
            headers=headers,
            cookies=cookies,
            method='GET',
            callback=self.parse
        )



    def parse(self, response):
        response_text = response.text
        # 1. 用正则提取 {...} 部分
        match = re.search(r"mtopjsonp22\((.*)\)", response_text)
        if match:
            json_str = match.group(1)
            phone_list = json.loads(json_str)["data"]["itemsArray"]
            # print(phone_list)  # 输出: {'key': 'value'}
            item = TaobaoItem()
            for phone in phone_list:
                # 去掉 文字中包含的标签
                item["title"] = re.sub(r'<[^>]+>', '', phone["title"])
                item["procity"] = phone["procity"]
                item["price"] = phone["price"]
                print(item)
                yield item




